import logging
from six.moves.urllib.parse import urljoin

from time import sleep

from scrapy.http import Request
from scrapy.utils.response import get_meta_refresh
from scrapy.exceptions import IgnoreRequest, NotConfigured

from scrapy.downloadermiddlewares.redirect import BaseRedirectMiddleware, MetaRefreshMiddleware, logger

from Hunter_Crawler.spiders.Jingdong import Jingdong_Review

logger = logging.getLogger(__name__)

class RedirectMiddleware(BaseRedirectMiddleware):
	"""Handle redirection of requests based on response status and meta-refresh html tag"""

	def process_response(self, request, response, spider):
		if request.meta.get('dont_redirect', False):
			return response

		if request.method == 'HEAD':
			if response.status in [301, 302, 303, 307] and 'Location' in response.headers:
				redirected_url = urljoin(request.url, response.headers['location'])
				redirected = request.replace(url=redirected_url)
				return self._redirect(redirected, request, spider, response.status)
			else:
				return response

		if response.status in [302, 303] and 'Location' in response.headers:
			redirected_url = urljoin(request.url, response.headers['location'])
			redirected = self._redirect_request_using_get(request, redirected_url)
			sleep(2.5)
			logger.debug('%(url)s redirected',{'url':response.url})

			if response.url.split('/')[2] == 'club.jd.com':
				return Request(response.url, callback=Jingdong_Review().parsepage, meta={'productID':request.meta['productID']})
			elif response.url.split('/')[2] == 'item.jd.com':
				return Request(response.url, callback=Jingdong_Product().parsepage, meta={'category':request.meta['category'],'productID':request.meta['productID'],'sale_rank':request.meta['sale_rank']})
#			return self._redirect(redirected, request, spider, response.status)

		if response.status in [301, 307] and 'Location' in response.headers:
			redirected_url = urljoin(request.url, response.headers['location'])
			redirected = request.replace(url=redirected_url)
			return self._redirect(redirected, request, spider, response.status)

		return response

